In [1]:
import numpy as np
import cv2,os

import matplotlib.pyplot as plt

from scipy.spatial.distance import cdist
from skimage.measure import label, regionprops

%matplotlib inline
In [2]:
dataNum = 0
dirNames = ["cfdn-f11/","cfdn-f22/","ox-1s/"]
rawImgPaths = [
    "/media/localuser/Windows/Users/Default/Desktop/datasets/cfdn-f11/",
    "/media/localuser/Windows/Users/Default/Desktop/datasets/cfdn-f22/",
    "../../../datasets/oxford/1-s/"
              ]

dataLoadPath = "../out/" + dirNames[dataNum]
rawImgPath = rawImgPaths[dataNum]
outDir = "../out/filtered-loop-track/" + dirNames[dataNum]

if not os.path.exists(outDir):
    os.makedirs(outDir)
    
imgCount = len(os.listdir(rawImgPath))

fileExtRaw = ".jpg"
if dataNum in [2]:
    fileExtRaw = ".png"
    
    
f0, xx0, yy0 = 983.044006, 643.646973, 493.378998
f0, xx0, yy0 = 0.8*f0, 0.8*xx0, 0.8*yy0
In [3]:
def loadProcImg(dataPath,idx,emb=False,colLab=False):
    img1 = cv2.imread(dataPath+"{0:07d}".format(idx)+fileExtRaw)
    
    binImg = img1[:,2*img1.shape[1]//4:3*img1.shape[1]//4]
    embImg = img1[:,img1.shape[1]//4:2*img1.shape[1]//4]
    colLabImg = img1[:,3*img1.shape[1]//4:]
    
    if emb and colLab:
        return binImg, embImg, colLabImg
    elif emb:
        return binImg, embImg
    elif colLab:
        return binImg, colLabImg
    else:
        return binImg

def loadRawImg(dataPath,idx):
    img1 = cv2.imread(dataPath+"{0:07d}".format(idx)+fileExtRaw)
    return img1#cv2.resize(img1,(512,256),interpolation=cv2.INTER_LINEAR)

def getLanePoints(idxOrImg,passedImg=False):
    if passedImg:
        im = idxOrImg.copy()        
    else:
        im = loadProcImg(dataLoadPath,idxOrImg)
    lanePts = np.where(im[:,:,0]==255)
    return np.array(lanePts).transpose()
    
tstIdx = 2
lanePts1 = getLanePoints(tstIdx)
plt.imshow(loadRawImg(rawImgPath,tstIdx))
plt.plot(lanePts1[:,1],lanePts1[:,0])
Out[3]:
[<matplotlib.lines.Line2D at 0x7f17b76c5780>]
In [4]:
# def getLabelsFromBinaryImg(inImg):
#     imL = label(inImg)
#     regData = regionprops(imL)
#     for i1 in range(len(regions)):
#         ...
#     # to do

def getLanesFromPoints(inPts,inImg):
    
    if inPts is None or inPts.shape[0] == 0:
        return None, None
    
    binImg = np.zeros(inImg.shape,np.uint8)
    binImg[inPts[:,0],inPts[:,1],:] = (255,255,255)

    procImg = cv2.dilate(binImg,np.ones((21,21),np.uint8))
    
    labImg = label(procImg)

    regions = regionprops(labImg)
    
    laneImg = np.zeros(inImg.shape,np.uint8)
    lanePointsAll = []
    for i1 in range(len(regions)):
        laneImgReg = np.zeros(inImg.shape,np.uint8)
        cords = regions[i1].coords
        y = cords[:,0]
        x = cords[:,1]

        fit = np.polyfit(x,y,1)
        fit_fn = np.poly1d(fit) 

        xnew = np.arange(x.min(),x.max())
        ynew = fit_fn(xnew).astype(int)

#         laneImg[ynew,xnew,:] = (255,255,255)
        laneImgReg = cv2.line(laneImgReg,(xnew[0],int(fit_fn(xnew[0]))),(xnew[-1],int(fit_fn(xnew[-1]))),
                          (255,255,255),2,cv2.LINE_AA)
    
        laneImg = cv2.line(laneImg,(xnew[0],int(fit_fn(xnew[0]))),(xnew[-1],int(fit_fn(xnew[-1]))),
                          (255,255,255),2,cv2.LINE_AA)
        
        lanePtsReg = np.array(np.where(laneImgReg[:,:,0]==255)).transpose()
        print(lanePtsReg.shape)
        
        lanePointsAll.append(lanePtsReg)
        
    lanePointsAll = np.concatenate(lanePointsAll)
    return laneImg, lanePointsAll

# idx = 7
# img1 = loadRawImg(rawImgPath,idx)
# prevPoints = getProjectedPoints(idx,idx+1)
# print(prevPoints.shape) 

# laneImg, lanePoints = getLanesFromPoints(prevPoints,img1)
# plt.imshow(laneImg)
In [5]:
colorsDefLoc = np.array([np.array([255, 0, 0]),
                           np.array([0, 255, 0]),
                           np.array([0, 0, 255]),
                           np.array([255, 255, 0]),
                           np.array([0, 255, 255]),
                           np.array([255, 0, 255]),
                           np.array([255, 255, 255]),
#                            np.array([125, 125, 125])
                        ])
    
def getLabelFromEmbedding(embImg):

    im2 = np.argmin(np.sum(abs(embImg[None] - colorsDefLoc[:,None,None,:]),axis=-1),axis=0).astype(np.uint8)
    return im2


def verifyPointEmbedding(pts,inEmbImg):
    embImg = inEmbImg.copy()
    embImg[:,:,2] = 0 # last channel is background
    embImg = np.max(embImg,axis=2) # channel color doesn't matter here
    embImg[embImg<embImg.mean()+2*embImg.std()] = 0 # some thresholding to get good embeddings (rest set to 0)
#     plt.imshow(embImg)
#     plt.colorbar()
#     plt.show()

    mutualPtsIdx = np.argwhere(embImg[pts[:,0],pts[:,1]] > 0)[:,0] # keep only those points that have +ve embeddings
    
    return mutualPtsIdx
def getLanesFromLabelledPoints(inPts,inLabels,inImg):
    uniLabels = np.unique(inLabels)
    laneImg = np.zeros(inImg.shape,np.uint8)
    lanePointsAll = []
    for lab in uniLabels:
        if lab==2:
            continue
        col1 = colorsDefLoc[lab]
        laneImgReg = np.zeros(inImg.shape,np.uint8)

        inds = np.where(inLabels==lab)
        cords = inPts[inds]

        y = cords[:,0]
        x = cords[:,1]
                
#         if len(cords) < 100:
#             continue

        fit = np.polyfit(x,y,1)
        fit_fn = np.poly1d(fit) 

        xnew = np.arange(x.min(),x.max())
        ynew = fit_fn(xnew).astype(int)
    
        if xnew.shape[0] == 0:
            continue
        
        laneImgReg = cv2.line(laneImgReg,(xnew[0],int(fit_fn(xnew[0]))),(xnew[-1],int(fit_fn(xnew[-1]))),
                          (255,255,255),2,cv2.LINE_AA)
        
        lanePtsReg = np.array(np.where(laneImgReg[:,:,0]==255)).transpose()
        
        goodInds = verifyPointEmbedding(lanePtsReg,inImg)
        if len(goodInds) < 0.5*len(lanePtsReg):
            continue
        
        lanePointsAll.append(lanePtsReg)
        laneImg = cv2.line(laneImg,(xnew[0],int(fit_fn(xnew[0]))),(xnew[-1],int(fit_fn(xnew[-1]))),
                          (int(col1[0]),int(col1[1]),int(col1[2])),2,cv2.LINE_AA)
        
    if len(lanePointsAll) != 0:   
        lanePointsAll = np.concatenate(lanePointsAll)
    else:
        lanePointsAll = None
#     newLanePts = np.array(np.where(laneImg==255)).transpose()
    
    return laneImg, lanePointsAll
In [6]:
def trackPoints_opFlow(imRaw1,imRaw2,inLanePts1):
    
    imC1 = imRaw1#[300:500]
    imC2 = imRaw2#[300:500]
    
    flowImg = cv2.calcOpticalFlowFarneback(cv2.cvtColor(imC1,cv2.COLOR_BGR2GRAY),
                             cv2.cvtColor(imC2,cv2.COLOR_BGR2GRAY),
                             None,0.5,3,11,100,5,1.1,cv2.OPTFLOW_FARNEBACK_GAUSSIAN)
    
    inPts1 = inLanePts1.astype(int).copy()
#     inPts1 = inLanePts1 - [300,0]
#     print(inPts1)
    
    lp_flow = flowImg[inPts1[:,0],inPts1[:,1]]
    outPts2 = inPts1 + np.fliplr(lp_flow)

#     plt.figure(figsize=(15,10))
#     showImg=getCorresImg(imC1,imC2,list(zip(inPts1,outPts2)),True,1)    
#     plt.show()

#     return inPts1 + [300,0], outPts2 + [300,0]
    return outPts2

def trackPoints_readOpFlowData(imIdx,inLanePts1):
    flowImg = cv2.imread("../../UnFlow/out/CSS/"+"{0:06d}".format(imIdx)+"_flow.png",
                     cv2.IMREAD_UNCHANGED)[:,:,:2].astype(float)-500
    
    inPts1 = inLanePts1.astype(int).copy()
#     inPts1 = inLanePts1 - [300,0]
#     print(inPts1)
    
    lp_flow = flowImg[inPts1[:,0],inPts1[:,1]]
    outPts2 = inPts1 + np.fliplr(lp_flow)
    
    return outPts2

def getUnion2D(ptSet1,ptSet2):
    disMat = cdist(ptSet1,ptSet2)
    numNonZeroCols = np.count_nonzero(disMat,axis=0)
    inds = np.argwhere(numNonZeroCols==disMat.shape[0])[:,0]
    return inds#np.concatenate([ptSet1,ptSet2[inds]])
    
def getProjectedPoints(idx1,idx2,prevPoints=None,prevLabels=None):

    im1 = loadRawImg(rawImgPath,idx1)
    im2 = loadRawImg(rawImgPath,idx2)
    im1B,im1E,im1L = loadProcImg(dataLoadPath,idx1,True,True) 
    im2B,im2E,im2L = loadProcImg(dataLoadPath,idx2,True,True) 

    lanePts1 = getLanePoints(im1B,passedImg=True)
    labelImg1 = getLabelFromEmbedding(im1E)
    labels1 = labelImg1[lanePts1[:,0],lanePts1[:,1]]
    
    lanePts2 = getLanePoints(im2B,passedImg=True)
    labelImg2 = getLabelFromEmbedding(im2E)
        
    if prevPoints is not None:
        unionInds = getUnion2D(lanePts1,prevPoints)
        lanePts1 = np.concatenate([lanePts1,prevPoints[unionInds]])
        labels1 = np.concatenate([labels1,prevLabels[unionInds]])
        
#     plt.figure(figsize=(15,10))
#     plt.imshow(cv2.cvtColor(im2,cv2.COLOR_BGR2RGB),)
    displayImg = im2.copy()
    laneImg = np.zeros(displayImg.shape,displayImg.dtype)
    allPointImg = laneImg.copy()
    print(lanePts1.shape)
    fittedPoints, fittedLabels = None, None
    if lanePts1.shape[0] != 0:
    
    #     trackedPts2 = trackPoints_opFlow(im1,im2,lanePts1)
        trackedPts2 = trackPoints_readOpFlowData(idx1,lanePts1)
        trackedPts2 = np.floor(trackedPts2).astype(int)

        boundedInds = np.where((trackedPts2<im2E.shape[:2]).prod(axis=1)==1)[0] # point indices that are beyond image boundary
        trackedPts2 = trackedPts2[boundedInds]
    
        intersectInds = verifyPointEmbedding(trackedPts2,im2E)
        mutualPts2 = trackedPts2[intersectInds].astype(int)
        labels2_ =  labelImg2[mutualPts2[:,0],mutualPts2[:,1]]

        allPointImg[mutualPts2[:,0],mutualPts2[:,1]] = colorsDefLoc[labels2_]#[255,255,255]
        laneImg, fittedPoints = getLanesFromLabelledPoints(mutualPts2,labels2_,im2E)#getLanesFromPoints(mutualPts2,im2)
        if fittedPoints is not None:
            fittedLabels = labelImg2[fittedPoints[:,0],fittedPoints[:,1]]
        
##         plt.scatter(mutualPts2[:,1],mutualPts2[:,0],c=mutualPtsColor/255.0)
#         plt.plot(fittedPoints[:,1],fittedPoints[:,0],'g.')  
            displayImg[fittedPoints[:,0],fittedPoints[:,1]] = [0,255,0]
     
    displayImg[lanePts2[:,0],lanePts2[:,1]] = [0,0,255]
        
#     plt.plot(lanePts2[:,1],lanePts2[:,0],'r.')

#     plt.gca().axis("off")    
#     plt.tight_layout()
#     plt.savefig(outDir+str(idx2)+".jpg",bbox_inches='tight', pad_inches = 0)
#     plt.close()  
    cv2.imwrite(outDir+str(idx2)+".jpg",np.hstack([displayImg,allPointImg,laneImg]))
    
    return fittedPoints, fittedLabels
In [7]:
# idx1, idx2 = 15,16
# pts2 = getProjectedPoints(idx1,idx2)

prevPoints = getLanePoints(0,passedImg=False)
im1B,im1E,im1L = loadProcImg(dataLoadPath,0,True,True) 
labelImg = getLabelFromEmbedding(im1E)
prevLabels = labelImg[prevPoints[:,0],prevPoints[:,1]]
    
for i1 in range(imgCount-1):
    print(i1)
    prevPoints, prevLabels = getProjectedPoints(i1,i1+1,prevPoints,prevLabels)
0
(0, 2)
1
(0, 2)
2
(845, 2)
3
(1109, 2)
4
(1641, 2)
5
(1436, 2)
6
(2097, 2)
7
(2538, 2)
8
(2468, 2)
9
(2635, 2)
10
(2678, 2)
11
(3558, 2)
12
(2713, 2)
13
(2376, 2)
14
(2183, 2)
15
(3163, 2)
16
(2104, 2)
17
(1739, 2)
18
(2229, 2)
19
(2830, 2)
20
(2041, 2)
21
(1924, 2)
22
(1809, 2)
23
(2262, 2)
24
(1958, 2)
25
(1611, 2)
26
(2446, 2)
27
(3516, 2)
28
(3466, 2)
29
(2744, 2)
30
(2480, 2)
31
(3469, 2)
32
(2308, 2)
33
(2037, 2)
34
(1224, 2)
35
(1804, 2)
36
(1073, 2)
37
(948, 2)
38
(492, 2)
39
(383, 2)
40
(880, 2)
41
(985, 2)
42
(1734, 2)
43
(2912, 2)
44
(2334, 2)
45
(3014, 2)
46
(4395, 2)
47
/home/localuser/miniconda2/envs/s_local_p3_keras_temp/lib/python3.5/site-packages/ipykernel_launcher.py:48: RankWarning: Polyfit may be poorly conditioned
(3417, 2)
48
(3000, 2)
49
(3459, 2)
50
(1868, 2)
51
(1537, 2)
52
(2996, 2)
53
(3007, 2)
54
(1805, 2)
55
(970, 2)
56
(1424, 2)
57
(2083, 2)
58
(1693, 2)
59
(2142, 2)
60
(2051, 2)
61
(966, 2)
62
(1000, 2)
63
(951, 2)
64
(1177, 2)
65
(2174, 2)
66
(2870, 2)
67
(2737, 2)
68
(828, 2)
69
(1402, 2)
70
(2079, 2)
71
(2434, 2)
72
(1422, 2)
73
(2033, 2)
74
(2290, 2)
75
(2601, 2)
76
(1781, 2)
77
(2359, 2)
78
(3462, 2)
79
(2919, 2)
80
(2758, 2)
81
(2661, 2)
82
/home/localuser/miniconda2/envs/s_local_p3_keras_temp/lib/python3.5/site-packages/ipykernel_launcher.py:48: RankWarning: Polyfit may be poorly conditioned
(3461, 2)
83
(2320, 2)
84
(2906, 2)
85
/home/localuser/miniconda2/envs/s_local_p3_keras_temp/lib/python3.5/site-packages/ipykernel_launcher.py:48: RankWarning: Polyfit may be poorly conditioned
(2688, 2)
86
(2653, 2)
87
(1866, 2)
88
(2004, 2)
89
(2652, 2)
90
(3304, 2)
91
(2212, 2)
92
(2332, 2)
93
/home/localuser/miniconda2/envs/s_local_p3_keras_temp/lib/python3.5/site-packages/ipykernel_launcher.py:48: RankWarning: Polyfit may be poorly conditioned
(2897, 2)
94
(3283, 2)
95
(1744, 2)
96
(1883, 2)
97
(3188, 2)
98
(1175, 2)
99
(1054, 2)
100
(610, 2)
101
(1182, 2)
102
(1417, 2)
103
(408, 2)
104
(203, 2)
105
(839, 2)
106
(2142, 2)
107
(1376, 2)
108
(1894, 2)
109
(2163, 2)
110
(2618, 2)
111
(2321, 2)
112
(1603, 2)
113
(1568, 2)
114
(2148, 2)
115
/home/localuser/miniconda2/envs/s_local_p3_keras_temp/lib/python3.5/site-packages/ipykernel_launcher.py:48: RankWarning: Polyfit may be poorly conditioned
(2459, 2)
116
(2807, 2)
117
(2359, 2)
118
(1930, 2)
119
(2809, 2)
120
(2495, 2)
121
(2740, 2)
122
(2787, 2)
123
(2213, 2)
124
(1537, 2)
125
(969, 2)
126
(1250, 2)
127
(1545, 2)
128
(1365, 2)
129
(2520, 2)
130
(2237, 2)
131
(2011, 2)
132
(1571, 2)
133
(1401, 2)
134
(585, 2)
135
(628, 2)
136
(764, 2)
137
(763, 2)
138
(1055, 2)
139
(1633, 2)
140
(2121, 2)
141
(1465, 2)
142
(1555, 2)
143
(1748, 2)
144
(2701, 2)
145
(2435, 2)
146
(2398, 2)
147
(1629, 2)
148
(1679, 2)
149
(1594, 2)
150
(1555, 2)
151
(3044, 2)
152
(2075, 2)
153
(1938, 2)
154
(818, 2)
155
(1150, 2)
156
(725, 2)
157
(657, 2)
158
(633, 2)
159
(584, 2)
160
(150, 2)
161
(135, 2)
162
(136, 2)
163
(136, 2)
164
(140, 2)
165
(142, 2)
166
(124, 2)
167
(763, 2)
168
(447, 2)
169
(446, 2)
170
(896, 2)
171
(1033, 2)
172
(1215, 2)
173
(1341, 2)
174
(1274, 2)
175
(859, 2)
176
(1292, 2)
177
(871, 2)
178
(466, 2)
179
(466, 2)
180
(464, 2)
181
(446, 2)
182
(1592, 2)
183
(1288, 2)
184
(1245, 2)
185
(1269, 2)
186
(876, 2)
187
(876, 2)
188
(860, 2)
189
(884, 2)
190
(857, 2)
191
(411, 2)
192
(1094, 2)
193
(456, 2)
194
(966, 2)
195
(462, 2)
196
(462, 2)
197
(1535, 2)
198
(853, 2)
199
(854, 2)
200
(855, 2)
201
(849, 2)
202
(850, 2)
203
(818, 2)
204
(851, 2)
205
(395, 2)
206
(395, 2)
207
(1082, 2)
208
(1376, 2)
209
(1218, 2)
210
(1263, 2)
211
(1213, 2)
212
(616, 2)
213
(1009, 2)
214
(448, 2)
215
(300, 2)
216
(254, 2)
217
(65, 2)
218
(0, 2)
219
(0, 2)
220
(0, 2)
221
(0, 2)
222
(0, 2)
223
(0, 2)
224
(0, 2)
225
(0, 2)
226
(0, 2)
227
(0, 2)
228
(0, 2)
229
(590, 2)
230
(789, 2)
231
(1004, 2)
232
(1082, 2)
233
(764, 2)
234
(298, 2)
235
(1167, 2)
236
(588, 2)
237
(1841, 2)
238
(2833, 2)
239
(3018, 2)
240
(2964, 2)
241
(2360, 2)
242
(2821, 2)
243
(3341, 2)
244
(3591, 2)
245
(2850, 2)
246
(2848, 2)
247
(3395, 2)
248
(4200, 2)
249
(3283, 2)
250
(822, 2)
251
(1102, 2)
252
(2057, 2)
253
(1807, 2)
254
(1758, 2)
255
(1257, 2)
256
(2369, 2)
257
(3421, 2)
258
(3227, 2)
259
(1186, 2)
260
(978, 2)
261
(1448, 2)
262
(1000, 2)
263
(830, 2)
264
(464, 2)
265
(446, 2)
266
(1047, 2)
267
(1943, 2)
268
(672, 2)
269
(652, 2)
270
(2676, 2)
271
(1837, 2)
272
(1688, 2)
273
(1046, 2)
274
(1357, 2)
275
(1525, 2)
276
(1077, 2)
277
(636, 2)
278
(1376, 2)
279
(1666, 2)
280
(1525, 2)
281
(1228, 2)
282
(670, 2)
283
(1360, 2)
284
(2273, 2)
285
(1365, 2)
286
(2442, 2)
287
(2787, 2)
288
(2564, 2)
289
(1099, 2)
290
(831, 2)
291
(780, 2)
292
(1225, 2)
293
(902, 2)
294
(461, 2)
295
(403, 2)
296
(1634, 2)
297
(1422, 2)
298
(1228, 2)
299
(542, 2)
300
(275, 2)
301
(1192, 2)
302
(838, 2)
303
(494, 2)
304
(493, 2)
305
(2314, 2)
306
(2355, 2)
307
(1967, 2)
308
(876, 2)
309
/home/localuser/miniconda2/envs/s_local_p3_keras_temp/lib/python3.5/site-packages/ipykernel_launcher.py:48: RankWarning: Polyfit may be poorly conditioned
(407, 2)
310
(28, 2)
311
(0, 2)
312
(0, 2)
313
(0, 2)
314
(0, 2)
315
(0, 2)
316
(0, 2)
317
(0, 2)
318
(0, 2)
319
(0, 2)
320
(611, 2)
321
(0, 2)
322
(0, 2)
323
(0, 2)
324
(0, 2)
325
(0, 2)
326
(0, 2)
327
(0, 2)
328
(0, 2)
329
(0, 2)
330
(0, 2)
331
(847, 2)
332
(1396, 2)
333
(1257, 2)
334
(601, 2)
335
(242, 2)
336
(0, 2)
337
(1013, 2)
338
(690, 2)
339
(692, 2)
340
(1478, 2)
341
(1415, 2)
342
(1776, 2)
343
(2302, 2)
344
(2486, 2)
345
(1620, 2)
346
(2586, 2)
347
(2638, 2)
348
(2706, 2)
349
(1975, 2)
350
(1726, 2)
351
(2228, 2)
352
(2656, 2)
353
(2147, 2)
354
(2732, 2)
355
(3253, 2)
356
(3462, 2)
357
(1758, 2)
358
(2344, 2)
359
(2509, 2)
360
(2103, 2)
361
(1632, 2)
362
(2495, 2)
363
(2377, 2)
364
(2496, 2)
365
(1778, 2)
366
(2715, 2)
367
(2689, 2)
368
(2759, 2)
369
(1864, 2)
370
(2758, 2)
371
(2519, 2)
372
(2313, 2)
373
(1607, 2)
374
(1804, 2)
375
(1737, 2)
376
(2183, 2)
377
(1710, 2)
378
(2767, 2)
379
(2529, 2)
380
/home/localuser/miniconda2/envs/s_local_p3_keras_temp/lib/python3.5/site-packages/ipykernel_launcher.py:48: RankWarning: Polyfit may be poorly conditioned
(2825, 2)
381
(2866, 2)
382
(2848, 2)
383
(2872, 2)
384
(3109, 2)
385
(3070, 2)
386
(2924, 2)
387
(2936, 2)
388
(2402, 2)
389
(4300, 2)
390
(3292, 2)
391
(3026, 2)
392
(1989, 2)
393
(3232, 2)
394
(2791, 2)
395
(3003, 2)
396
(4148, 2)
397
(3832, 2)
398
(3601, 2)
399
(1362, 2)
400
(2417, 2)
401
(1543, 2)
402
(1294, 2)
403
(1783, 2)
404
(3595, 2)
405
(2419, 2)
406
(2879, 2)
407
(2698, 2)
408
(2736, 2)
409
(2430, 2)
410
(3454, 2)
411
(2392, 2)
412
(3381, 2)
413
(4181, 2)
414
(2480, 2)
415
(3523, 2)
416
(3895, 2)
417
(3084, 2)
418
(2472, 2)
419
(2547, 2)
420
(3859, 2)
421
(2497, 2)
422
(2732, 2)
423
(2982, 2)
424
(2804, 2)
425
(2494, 2)
426
(3012, 2)
427
/home/localuser/miniconda2/envs/s_local_p3_keras_temp/lib/python3.5/site-packages/ipykernel_launcher.py:48: RankWarning: Polyfit may be poorly conditioned
(2778, 2)
428
(1922, 2)
429
(2593, 2)
430
(2595, 2)
431
(2605, 2)
432
(1824, 2)
433
(3132, 2)
434
(2902, 2)
435
(1702, 2)
436
(1499, 2)
437
(1645, 2)
438
(2313, 2)
439
(1388, 2)
440
(1148, 2)
441
(1947, 2)
442
(1528, 2)
443
(1356, 2)
444
(2282, 2)
445
(2170, 2)
446
(877, 2)
447
(876, 2)
448
(92, 2)
449
(732, 2)
450
(1911, 2)
451
(2275, 2)
452
(2253, 2)
453
(1149, 2)
454
(0, 2)
455
(0, 2)
456
(0, 2)
457
(0, 2)
458
(0, 2)
459
(0, 2)
460
(0, 2)
461
(0, 2)
462
(617, 2)
463
(2928, 2)
464
(3504, 2)
465
(3930, 2)
466
(2456, 2)
467
(1490, 2)
468
(1398, 2)
469
(2895, 2)
470
(2457, 2)
471
(1437, 2)
472
(1447, 2)
473
(1334, 2)
474
(1166, 2)
475
(1044, 2)
476
(1069, 2)
477
(1782, 2)
478
(914, 2)
479
(904, 2)
480
(1645, 2)
481
(2242, 2)
482
(1340, 2)
483
(1253, 2)
484
(2022, 2)
485
(1937, 2)
486
(1326, 2)
487
(1279, 2)
488
(3180, 2)
489
(2124, 2)
490
(929, 2)
491
(1565, 2)
492
(2497, 2)
493
(1394, 2)
494
(1930, 2)
495
(1435, 2)
496
/home/localuser/miniconda2/envs/s_local_p3_keras_temp/lib/python3.5/site-packages/ipykernel_launcher.py:48: RankWarning: Polyfit may be poorly conditioned
(1508, 2)
497
(1214, 2)
498
(1288, 2)
499
/home/localuser/miniconda2/envs/s_local_p3_keras_temp/lib/python3.5/site-packages/ipykernel_launcher.py:48: RankWarning: Polyfit may be poorly conditioned
(2160, 2)
500
(1856, 2)
501
(1409, 2)
502
(2499, 2)
503
(3137, 2)
504
(1472, 2)
505
(1477, 2)
506
(3045, 2)
507
(2877, 2)
508
(1498, 2)
509
(2013, 2)
510
(2809, 2)
511
(2001, 2)
512
(1880, 2)
513
(1497, 2)
514
(2243, 2)
515
(1452, 2)
516
(1152, 2)
517
(1888, 2)
518
(2287, 2)
519
(760, 2)
520
(500, 2)
521
(440, 2)
522
(351, 2)
523
(244, 2)
524
(187, 2)
525
(1031, 2)
526
(1034, 2)
527
(93, 2)
528
(98, 2)
529
(1705, 2)
530
(1622, 2)
531
(1268, 2)
532
(1023, 2)
533
(2748, 2)
534
(1956, 2)
535
(1236, 2)
536
(1348, 2)
537
(1494, 2)
538
(1812, 2)
539
(2322, 2)
540
(2837, 2)
541
(3011, 2)
542
(2390, 2)
543
(2973, 2)
544
(1173, 2)
545
(2256, 2)
546
(2629, 2)
547
(2000, 2)
548
(1415, 2)
549
/home/localuser/miniconda2/envs/s_local_p3_keras_temp/lib/python3.5/site-packages/ipykernel_launcher.py:48: RankWarning: Polyfit may be poorly conditioned
(1683, 2)
550
(981, 2)
551
/home/localuser/miniconda2/envs/s_local_p3_keras_temp/lib/python3.5/site-packages/ipykernel_launcher.py:48: RankWarning: Polyfit may be poorly conditioned
(827, 2)
552
(819, 2)
553
/home/localuser/miniconda2/envs/s_local_p3_keras_temp/lib/python3.5/site-packages/ipykernel_launcher.py:48: RankWarning: Polyfit may be poorly conditioned
(995, 2)
554
(498, 2)
555
(799, 2)
556
(979, 2)
557
(2220, 2)
558
(2230, 2)
559
(1564, 2)
560
(3369, 2)
561
(3536, 2)
562
(2357, 2)
563
(1766, 2)
564
(2225, 2)
565
(838, 2)
566
(474, 2)
567
(277, 2)
568
(998, 2)
569
(1092, 2)
570
(348, 2)
571
(1389, 2)
572
(1885, 2)
573
(1943, 2)
574
(1447, 2)
575
(2371, 2)
576
(1442, 2)
577
(1280, 2)
578
(359, 2)
579
(1183, 2)
580
(1985, 2)
581
(1573, 2)
582
(1061, 2)
583
(657, 2)
584
(578, 2)
585
/home/localuser/miniconda2/envs/s_local_p3_keras_temp/lib/python3.5/site-packages/ipykernel_launcher.py:48: RankWarning: Polyfit may be poorly conditioned
(554, 2)
586
/home/localuser/miniconda2/envs/s_local_p3_keras_temp/lib/python3.5/site-packages/ipykernel_launcher.py:48: RankWarning: Polyfit may be poorly conditioned
(1697, 2)
587
(1848, 2)
588
(3054, 2)
589
(2569, 2)
590
/home/localuser/miniconda2/envs/s_local_p3_keras_temp/lib/python3.5/site-packages/ipykernel_launcher.py:48: RankWarning: Polyfit may be poorly conditioned
(2324, 2)
591
(1575, 2)
592
(2980, 2)
593
(2283, 2)
594
(2532, 2)

Compare output

In [315]:
outDirComp = outDir + "compare/"
if not os.path.exists(outDirComp):
    os.makedirs(outDirComp)
    
for i1 in range(imgCount):
    
    im1 = loadRawImg(rawImgPath,i1)
    im1B = loadProcImg(dataLoadPath,i1) 
    lanePts1 = getLanePoints(im1B,passedImg=True)
    
    plt.figure(figsize=(15,10))
    plt.imshow(cv2.cvtColor(im1,cv2.COLOR_BGR2RGB),)
    plt.plot(lanePts1[:,1],lanePts1[:,0],'r.')
    
    plt.gca().axis("off")    
    plt.tight_layout()
    plt.savefig("./tmp.jpg",bbox_inches='tight', pad_inches = 0)
    plt.close()     
    
    imgTmp = cv2.imread("./tmp.jpg")
    imgNew = cv2.imread(outDir+str(i1)+".jpg")
    if imgNew is None:
        imgNew = imgTmp
        
    imgFinal = np.concatenate([imgTmp,imgNew],axis=0)
    
    cv2.imwrite(outDirComp+str(i1)+".jpg",imgFinal)
    
    
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-315-9f428323e29b> in <module>()
     23         imgNew = imgTmp
     24 
---> 25     imgFinal = np.concatenate([imgTmp,imgNew],axis=0)
     26 
     27     cv2.imwrite(outDirComp+str(i1)+".jpg",imgFinal)

ValueError: all the input array dimensions except for the concatenation axis must match exactly
In [53]:
# track within a pair

def getCorresImg(img1,img2,matchingCords,show=False,circOrRec=0,scaleFac=1):
    
    color = plt.cm.rainbow(np.linspace(0,1,len(matchingCords)))
    outImg = np.concatenate([img1,img2],axis=1)
    for i in range(len(matchingCords)):
        ptLoc1 = (matchingCords[i][0]).astype(int)
        ptLoc2 = (matchingCords[i][1]).astype(int)
        
        ptLoc2Shifted = ptLoc2.copy()
        ptLoc2Shifted[1] += img1.shape[1]

        if circOrRec == 0:
            cv2.circle(outImg,(ptLoc1[1],ptLoc1[0]),1,(255,255,255),1,cv2.LINE_AA)
            cv2.circle(outImg,(ptLoc2Shifted[1],ptLoc2Shifted[0]),1,(255,255,255),1,cv2.LINE_AA)
        else:
            cv2.rectangle(outImg,(ptLoc1[1]-scaleFac,ptLoc1[0]-scaleFac),(ptLoc1[1]+scaleFac,ptLoc1[0]+scaleFac),255*color[i],2,cv2.LINE_AA)
            cv2.rectangle(outImg,(ptLoc2Shifted[1]-scaleFac,ptLoc2Shifted[0]-scaleFac),(ptLoc2Shifted[1]+scaleFac,ptLoc2Shifted[0]+scaleFac),255*color[i],2,cv2.LINE_AA)            

        cv2.line(outImg,(ptLoc1[1],ptLoc1[0]),(ptLoc2Shifted[1],ptLoc2Shifted[0]),255*color[i],2,cv2.LINE_AA)

    if show:
        plt.imshow(outImg)
        plt.show()
    return outImg


def getFeatureCorresConventional(im1_,im2_):

    orb = cv2.ORB_create()#(nfeatures=5000,scaleFactor=1.2,nlevels=5,edgeThreshold=1,patchSize=20)
#     sift = cv2.ORB_create(nfeatures=5000)#,edgeThreshold=1,patchSize=20)
    
    # find the keypoints and descriptors with SIFT
    kp1, des1 = orb.detectAndCompute(im1_,None)
    kp2, des2 = orb.detectAndCompute(im2_,None)

    
#     # FLANN parameters
#     FLANN_INDEX_KDTREE = 0
#     index_params = dict(algorithm = FLANN_INDEX_KDTREE, trees = 5)
#     search_params = dict(checks=50)
    
    flann = cv2.BFMatcher()#index_params,search_params)
    matches = flann.knnMatch(des1,des2,k=2)
    
    good = []
    pts1 = []
    pts2 = []
    
    # ratio test as per Lowe's paper
    for i,(m,n) in enumerate(matches):
        if m.distance < 0.7*n.distance:
            good.append(m)
            pts2.append(kp2[m.trainIdx].pt)
            pts1.append(kp1[m.queryIdx].pt) 
            
    pts1 = np.array(pts1)
    pts2 = np.array(pts2)

    showImg = None
    plt.figure(figsize=(15,10))
    showImg=getCorresImg(im1,im2,list(zip(np.fliplr(pts1),np.fliplr(pts2))),True,1)    
    plt.show()
    
    return pts1, pts2, showImg
    
def trackPair(binImg1,binImg2,rawImg1,rawImg2):
    
    # get pose between raw images # (can also try embedding images for this purpose for robustness)
    
    # get pixels to track from the binary images
    
    # project pixels to the other image using the pose
    
    # display
    return
In [115]:
im1 = loadRawImg(rawImgPath,0)
im2 = loadRawImg(rawImgPath,1)
pts1, pts2, _ = getFeatureCorresConventional(im1,im2)

Essential Matrix (Epipolar Geometry)

In [69]:
# E,mask = cv2.findEssentialMat(pts2, pts1,)# focal, pp, RANSAC, 0.999, 1.0, mask);
# numPoints,R,t,mask = cv2.recoverPose(E, pts2, pts1,)# R, t, focal, pp, mask);
E,mask = cv2.findEssentialMat(pts2, pts1, f0, (xx0,yy0), cv2.RANSAC, 0.999, 1.0)
numPoints,R,t,mask = cv2.recoverPose(E, pts2, pts1, focal=f0, pp=(xx0,yy0),mask=mask);
print(numPoints,R,t,mask.sum())
mask = mask.astype(bool)
def projPoints(pts,R_,t_):
    ptsH = cv2.convertPointsToHomogeneous(pts)[:,0,:]
    return cv2.convertPointsFromHomogeneous(np.matmul(ptsH,R_) + t_.transpose())[:,0,:]
pts1_ = projPoints(pts1,R,t)
print(pts1_.shape)
plt.figure(figsize=(15,10))
showImg=getCorresImg(im1,im2,list(zip(np.fliplr(pts1[mask[:,0]]),np.fliplr(pts1_[mask[:,0]]))),True,1)    
plt.show()
16 [[ 0.97980228  0.02242494  0.19870735]
 [-0.01861034  0.99960532 -0.02104419]
 [-0.19910084  0.01692113  0.97983291]] [[ 0.33605749]
 [-0.0402792 ]
 [ 0.94097978]] 16
(23, 2)
In [150]:
projPoints(np.array([[550,375],[600,425]]),R,t)
Out[150]:
array([[556.43789813, 384.94541555],
       [607.59104961, 436.56592012]])

Homography

In [135]:
M, mask = cv2.findHomography(pts1, pts2, cv2.RANSAC,5.0)
matchesMask = mask.ravel().tolist()

plt.figure(figsize=(15,10))
showImg=getCorresImg(im1,im2,list(zip(np.fliplr(pts1[mask[:,0]]),np.fliplr(pts2[mask[:,0]]))),True,1)    
plt.show()

h,w,_ = im1.shape
pts = np.float32([ [0,0],[0,h-1],[w-1,h-1],[w-1,0],[550,375],[600,425] ]).reshape(-1,1,2)
dst = cv2.perspectiveTransform(pts,M)

img2 = cv2.polylines(im2.copy(),[np.int32(dst)],True,255,3, cv2.LINE_AA)
plt.imshow(img2)
Out[135]:
<matplotlib.image.AxesImage at 0x7f919830e278>

Image Alignment

In [232]:
_, W = cv2.findTransformECC(cv2.cvtColor(im1[300:500],cv2.COLOR_BGR2GRAY),
                     cv2.cvtColor(im2[300:500],cv2.COLOR_BGR2GRAY),
                     np.eye(3,dtype=np.float32),motionType=cv2.MOTION_HOMOGRAPHY)
In [233]:
h,w,_ = im1.shape
pts = np.float32([ [0,0],[0,h-1],[w-1,h-1],[w-1,0],[550,75],[600,125] ]).reshape(-1,1,2)
dst = np.matmul(W,cv2.convertPointsToHomogeneous(pts[:,0,:])[:,0,:].transpose())[:2,:].transpose()#cv2.perspectiveTransform(pts,W)
dst = np.matmul(W,np.array([[550,75,1],[600,125,1]]).transpose()).transpose()[:2,:2]
img1 = cv2.polylines(im1[300:500].copy(),[np.int32([[550,75],[600,125]])],True,255,3, cv2.LINE_AA)
plt.imshow(img1)
plt.show()
img2 = cv2.polylines(im2[300:500].copy(),[np.int32(dst)],True,255,3, cv2.LINE_AA)
plt.imshow(img2)
Out[233]:
<matplotlib.image.AxesImage at 0x7f917b7c6550>

Optical Flow

In [273]:
pts2_,stat,err = cv2.calcOpticalFlowPyrLK(im1,im2,lanePts1.astype(np.float32),None)
plt.figure(figsize=(15,10))
showImg=getCorresImg(im1,im2,list(zip(lanePts1,pts2_)),True,1)    
plt.show()
# print(stat,err)

Dense

In [117]:
flowImg = cv2.calcOpticalFlowFarneback(
#     cv2.cvtColor(im1[300:500],cv2.COLOR_BGR2GRAY),
#     cv2.cvtColor(im2[300:500],cv2.COLOR_BGR2GRAY),
    cv2.cvtColor(im1,cv2.COLOR_BGR2GRAY),
    cv2.cvtColor(im2,cv2.COLOR_BGR2GRAY),    
    None,0.5,3,31,100,5,1.1,cv2.OPTFLOW_FARNEBACK_GAUSSIAN)
In [110]:
def draw_flow(img, flow, step=16):
    h, w = img.shape[:2]
    y, x = np.mgrid[step/2:h:step, step/2:w:step].reshape(2,-1).astype(int)
    fx, fy = flow[y,x].T
    lines = np.vstack([x, y, x+fx, y+fy]).T.reshape(-1, 2, 2)
    lines = np.int32(lines + 0.5)
    vis = img.copy()#cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
    cv2.polylines(vis, lines, 0, (0, 255, 0))
    for (x1, y1), (_x2, _y2) in lines:
        cv2.circle(vis, (x1, y1), 1, (0, 255, 0), -1)
    return vis
In [118]:
# vis = draw_flow(im1[300:500],flowImg)
vis = draw_flow(im1,flowImg)
plt.figure(figsize=(15,10))
plt.imshow(vis)
plt.show()
In [348]:
%matplotlib inline
lp_flow = flowImg[lanePts1[:,0]-300,lanePts1[:,1]]
pts2_ = lanePts1-[300,0] + np.fliplr(lp_flow)
plt.figure(figsize=(15,10))
showImg=getCorresImg(im1[300:500],im2[300:500],list(zip(lanePts1-[300,0],pts2_)),True,1)    
plt.show()

Embeddings thresholding

In [39]:
%matplotlib inline
_, embImg = loadProcImg(dataLoadPath,2,True)
embImg[:,:,2] = 0
embImg[embImg<embImg.mean()+2*embImg.std()] = 0
plt.imshow(embImg)
plt.colorbar()
embImg.mean()
Out[39]:
0.7093799732349537

Load point labels

In [90]:
ptLabels = np.load(dataLoadPath+"0000002.jpg.npz")['arr_0']
print(ptLabels.max())
plt.imshow(ptLabels)
255
Out[90]:
<matplotlib.image.AxesImage at 0x7f4b63971f98>

Temp Misc

In [77]:
im1 = cv2.imread("../out/filtered-loop/cfdn-f11/11.jpg")
im2 = cv2.imread("../out/filtered-loop/cfdn-f11/12.jpg")
im3 = cv2.imread("../out/filtered-loop/cfdn-f11/13.jpg")

plt.figure(figsize=(30,10))
plt.imshow(np.hstack([im1,im2,im3]))
plt.tight_layout()
# plt.savefig("./trackedPoints.jpg")

Load deep dense optical flow

In [256]:
idx = 49
# flo = np.load("../../UnFlow/out/CSS/"+"{0:06d}".format(idx)+"_flow.png.npz")['arr_0'][0]
flo = cv2.imread("../../UnFlow/out/CSS/"+"{0:06d}".format(idx)+"_flow.png",cv2.IMREAD_UNCHANGED)[:,:,:2].astype(float)-500
# flo = np.loadtxt("../../flownet2-pytorch/work/inference/run.epoch-0-flow-field/000000.flo")
print(flo.shape)
(576, 1024, 2)
In [257]:
vis = draw_flow(loadRawImg(rawImgPath,idx),flo)
plt.figure(figsize=(15,10))
plt.imshow(vis)
plt.show()

plt.imshow(flo[:,:,0])
plt.colorbar()
plt.show()

plt.imshow(flo[:,:,1])
plt.colorbar()
plt.show()

Post process tracked points

In [339]:
import sys
sys.path.append('/home/localuser/workspace/lanenet-lane-detection/')

from lanenet_model import lanenet_postprocess
postprocessor = lanenet_postprocess.LaneNetPoseProcessor()
In [582]:
idx = 52
img1 = loadRawImg(rawImgPath,idx)
prevPoints = getProjectedPoints(idx,idx+1)
print(prevPoints.shape) 

binary_seg_image = np.zeros(img1.shape,np.uint8)
binary_seg_image[prevPoints[:,0],prevPoints[:,1],:] = (255,255,255)
plt.imshow(binary_seg_image)
plt.show()

procImg = postprocessor.postprocess(binary_seg_image)
plt.imshow(procImg)
plt.show()
(2638, 2)
(160, 2)
(610, 2)
(422, 2)
(258, 2)
(1450, 2)
In [468]:
out = label(procImg)
plt.imshow(255*out/float(out.max()))

regions = regionprops(out)
fitImg = np.zeros(img1.shape,np.uint8)
# plt.imshow(fitImg)

for i1 in range(len(regions)):
    cords = regions[i1].coords
    y = cords[:,0]
    x = cords[:,1]
    
    fit = np.polyfit(x,y,1)
    fit_fn = np.poly1d(fit) 
    
    xnew = np.arange(x.min(),x.max())
    ynew = fit_fn(xnew).astype(int)
        
    fitImg[ynew,xnew,:] = (255,255,255)

    fitImg = cv2.line(fitImg,(xnew[0],int(fit_fn(xnew[0]))),(xnew[-1],int(fit_fn(xnew[-1]))),
                      (255,255,255),2,cv2.LINE_AA)

#     plt.plot(xnew,ynew)
plt.imshow(fitImg)
# cv2.imwrite("../tmp.jpg",fitImg)
Out[468]:
True

Get label from color

In [588]:
colorsDef = np.array([np.array([255, 0, 0]),
                           np.array([0, 255, 0]),
                           np.array([0, 0, 255]),
                           np.array([125, 125, 0]),
                           np.array([0, 125, 125]),
                           np.array([125, 0, 125]),
                           np.array([50, 100, 50]),
                           np.array([100, 50, 100])])
colorCodeUni = np.sum(colorsDef*[100,10,1],axis=1)
print(colorCodeUni)
assert(len(colorCodeUni) == len(np.unique(colorCodeUni)))

codeConverter = np.zeros(np.max(colorCodeUni)*10,int)
codeConverter[colorCodeUni] = np.arange(1,len(colorCodeUni)+1)
print(np.unique(codeConverter,True))
[25500  2550   255 13750  1375 12625  6050 10600]
(array([0, 1, 2, 3, 4, 5, 6, 7, 8]), array([    0, 25500,  2550,   255, 13750,  1375, 12625,  6050, 10600]))
In [540]:
# %matplotlib inline
_,_,imColLab = loadProcImg(dataLoadPath,6,True,True) 
plt.imshow(imColLab)
plt.show()
# print(np.unique(imColLab))

# imColLab_Uni = np.sum(imColLab*np.array([100,10,1])[None,None,:],axis=2)
# plt.imshow(imColLab_Uni)
# plt.colorbar()
# plt.show()
# print(np.unique(imColLab_Uni).shape)

# imLabel = codeConverter[imColLab_Uni]
# plt.imshow(imLabel*100)
# plt.colorbar()
# plt.show()
# print(np.unique(imLabel))

From embeddings image

In [696]:
%matplotlib inline
colorsDef = np.array([np.array([255, 0, 0]),
                           np.array([0, 255, 0]),
                           np.array([0, 0, 255]),
                           np.array([255, 255, 0]),
                           np.array([0, 255, 255]),
                           np.array([255, 0, 255]),
                           np.array([255, 255, 255]),
                           np.array([0, 0, 0])])

# _,imEmb = loadProcImg(dataLoadPath,6,True,False) 
# plt.imshow(imEmb)
# plt.show()

im2 = np.argmin(np.sum(abs(imEmb[None] - colorsDef[:,None,None,:]),axis=-1),axis=0).astype(float)
# im2[im2!=7] = 7
plt.imshow(im2)
plt.colorbar()
plt.show()